package practice.library.webmagic.processor_test

import us.codecraft.webmagic.processor.PageProcessor
import us.codecraft.webmagic.{Page, Site}

/**
  * Created by lovepocky(lovepocky@foxmail.com) on 2016/6/16.
  */
class GithubRepoPageProcessorExample extends PageProcessor {
	val site = Site.me().setRetryTimes(3).setSleepTime(1000)
	
	override def getSite: Site = site
	
	override def process(page: Page): Unit = {
		page.addTargetRequests(
			page.getHtml.links().regex("(http://github\\.com/\\w+/\\w+)").all()
		)
		page.putField("author", page.getUrl.regex("https://github\\.com/(\\w+)/.*").toString)
		page.putField("name", page.getHtml.xpath("//h1[@class='entry-title public']/strong/a/text()").toString)
		if (page.getResultItems.get("name") == null) {
			page.setSkip(true)
		}
		page.putField("readme", page.getHtml.xpath("//div[@id='readme']/tidyText()"))
	}
}
